package org.mentalsmash.crossroads.web;

import java.io.FileNotFoundException;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.commons.httpclient.HttpClient;
import org.apache.log4j.Logger;
import org.mentalsmash.crossroads.Configuration;
import org.webharvest.definition.ScraperConfiguration;
import org.webharvest.exception.ScraperXPathException;
import org.webharvest.runtime.Scraper;
import org.webharvest.runtime.variables.Variable;

/**
 *Utility class for transforming html documents into an array of string of the desired length. For this purpose it makes uses
 *of the Web Harvest. See conf in conf/web-harvest.conf.
 * @author Andrea Reale
 *
 */
public class HtmlFilter{
	
	private static Logger log = Logger.getLogger(HtmlFilter.class);
	private int _wordLength;
	private String _filterRegExp;

	
	/**
	 * Create a filter given a number of results and the length of each word we want in filter's output
	 * @param numResults number of words to be given in filter's results
	 * @param wordLength length of each word in thre result set
	 */
	public HtmlFilter(int wordLength){
		
		//ensuring configuration is initialized
		Configuration.getInstance();
		setWordLength(wordLength);
		
		setFilterRegExp(buildFilterRegexp(wordLength));

	}


	private String buildFilterRegexp(int wordLength) {
		String regexp = "'([\\w\\d]{" +
			wordLength +
			"})\\b | \\b([\\w\\d]{" +
			wordLength +
			"})\\b | \\b\"([\\w\\d]{" +
			wordLength + 
			"})\"\\b ";
		return regexp;
	}
	


	/**
	 * The length of the words that the filter will serach for inside the body of htmls documents
	 * @return The length of the words that the filter will serach for inside the body of htmls documents
	 */
	public int getWordLength() {
		return _wordLength;
	}

	/**
	 * Set the length of the word that the method filter() will search for inside the passed html document
	 * @param wordLength the length of the words to search for
	 */
	public void setWordLength(int wordLength) {
		_wordLength = wordLength;
	}
	

	/**
	 * Set the regular expression that will be used by this filter to extract words from a page.
	 * @param filterRegExp a regexp
	 */
	public void setFilterRegExp(String filterRegExp) {
		_filterRegExp = filterRegExp;
	}


	/**
	 * Gets the regularexpression used from the filter to extract words from a page
	 * HINT: use this method to see what the default val is :)
	 * @return a string representing a regexp
	 */
	public String getFilterRegExp() {
		return _filterRegExp;
	}


	/**
	 * Filters the words on an html page retrieved from the http url passed as argument, according to the regular expression set on the filter.
	 * The default behaviur is to return the words of length n, where n is the _wordLength parameter 
	 * set when the instance of the class was built
	 * @param htmlDocumentUrl an HTTP url wich point to an html page source
	 * @return read the description buddy :)
	 */
	public String[] filter(URL htmlDocumentUrl)  throws FilterException{
		ScraperConfiguration config = null;	
		try {
			config = new ScraperConfiguration(Configuration.getInstance().getWebHarvestConf());
		} catch (FileNotFoundException e) {
			log.fatal("Web Harvest configuration file not found at " + Configuration.getInstance().getWebHarvestConf() + " " + e.getMessage());
			System.exit(-1);
		}
		Scraper webHarvest = new Scraper(config,Configuration.getInstance().getWebHarvestWorkDir());
		
		//configuring scraper params
		
		
		HttpClient whClient = webHarvest.getHttpClientManager().getHttpClient();

		whClient.getParams().setParameter("http.connection-manager.timeout", Configuration.getInstance().getHttpTimeout());
		whClient.getParams().setParameter("http.socket.timeout",(int)Configuration.getInstance().getHttpTimeout());
		
		
		webHarvest.addVariableToContext("documentUrl", htmlDocumentUrl.toExternalForm());
		webHarvest.addVariableToContext("regExpression", getFilterRegExp());
		//Disable this in future 
		//webHarvest.setDebug(true);
		try{
			webHarvest.execute();
		}catch (ScraperXPathException e){
			throw new FilterException(e);
		} catch(org.webharvest.exception.HttpException e){
			throw new FilterException(e);
		}
		Variable vocabulary = (Variable) webHarvest.getContext().getVar("vocabulary");
		
		Object[] objWords = vocabulary.toArray();
		String[] words = new String[objWords.length];
		
		for(int i=0; i<objWords.length;i++)
			words[i]=objWords[i].toString();
		
		return words;
	}
	

	/**
	 * Filters the words on an html page retrieved from the http url passed as argument, according to the regular expression set on the filter.
	 * The default behaviur is to return the words of length n, where n is the _wordLength parameter 
	 * set when the instance of the class was built
	 * @param htmlDocumentUrl an HTTP url wich point to an html page source
	 * @return read the description buddy :)
	 */
	public String[] filter(String htmlDocumentUrl) throws MalformedURLException,FilterException {
		URL toFilter = new URL(htmlDocumentUrl);
		
		return filter(toFilter);
	}
	


}
